#ifndef TOKENIZER_HPP
#define TOKENIZER_HPP

struct Token
{
	Token();
	Token(int type);
	Token(int type, const char *text, bool quoted);
	string toString() const;
	
	int type;
	string text;
	bool quoted;
};

struct Tokenization
{
public:
	Tokenization(ShellState *shellState);
	
	void addToken(int type);
	void addToken(int type, const char *text, bool quoted);
	
	ShellState *shellState;
	vector<Token> tokens;
	bool truncated;
	const char *end;
	string pendingToken;
};

void tokenize(const char *cmd, Tokenization *state, int context);
void interactiveTokenizerTest();
string tokenizationToString(const Tokenization *tokenization);
bool isSeparator(char ch);

#define CONTEXT_ROOT 0
#define CONTEXT_COMMANDEXPANSION 1
#define CONTEXT_BACKQUOTED 2

#endif
